﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Web;
using System.Net;
using System.IO;
using System.Text;
using System.Text.RegularExpressions;
using System.IO.Compression;

/// <summary>
///HtmlTool 的摘要说明
/// </summary>
public class HtmlTool
{
    /// <summary>
    /// 获取网页的HTML内容，指定Encoding
    /// </summary>
    /// <param name="strUrl"></param>
    /// <param name="encoding"></param>
    /// <returns></returns>
    public static string GetHtml(string strUrl)
    {
        string strHTML = string.Empty;
        try
        {
            string strDomain = GetWebUrl(strUrl);
            HttpWebRequest requestPage = (HttpWebRequest)WebRequest.Create(new Uri(strUrl));
            requestPage.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            requestPage.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)";
            requestPage.Method = "GET";
            requestPage.Referer = "http://" + strDomain;
            requestPage.Host = strDomain;
            requestPage.Headers.Add("Accept-Language", "zh-cn,zh;q=0.8,en-us;q=0.5,en;q=0.3");
            requestPage.Headers.Add("Accept-Encoding", "gzip,deflate");
            CookieContainer cc = new CookieContainer();
            requestPage.CookieContainer = cc;
            HttpWebResponse responsePage = (HttpWebResponse)requestPage.GetResponse();
            Stream receiveStream = responsePage.GetResponseStream();
            if (responsePage.ContentEncoding.ToLower().Contains("gzip"))
            {
                receiveStream = new GZipStream(receiveStream, CompressionMode.Decompress);
            }

            //获取网页字符编码描述信息 
            byte[] buffer = StreamToBytes(receiveStream);
            Encoding encode = Encoding.UTF8;
            strHTML = encode.GetString(buffer);
            string strEncod = string.Empty;
            Match charSetMatch = Regex.Match(strHTML, @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)", RegexOptions.IgnoreCase | RegexOptions.Multiline);
            if (charSetMatch.Success)
            {
                string webCharSet = charSetMatch.Groups["charset"].Value.Trim();
                strEncod = webCharSet.ToLower();
            }
            else
            {

                charSetMatch = Regex.Match(strHTML, "<meta([^<]*)charset=[\"'](?<charset>[-a-zA-Z_0-9]+)[\"']", RegexOptions.IgnoreCase | RegexOptions.Multiline);
                if (charSetMatch.Success)
                {
                    string webCharSet = charSetMatch.Groups["charset"].Value.Trim();
                    strEncod = webCharSet.ToLower();
                }
            }
            if (string.IsNullOrEmpty(strEncod))
            {
                strEncod = responsePage.CharacterSet.Trim().ToLower();
            }
            if (string.IsNullOrEmpty(strEncod))
            {
                strEncod = "utf-8";
            }
            if (strEncod != "utf-8" && strEncod != "utf8")
            {
                encode = Encoding.GetEncoding(strEncod);
                strHTML = encode.GetString(buffer);
            }
            receiveStream.Close();
            buffer = null;
            return strHTML.Trim();
        }
        catch (Exception ex)
        {
            Common.Log("获取页面失败:" + strUrl + " " + ex.Message.ToString());
            return "";
        }
    }

    /// <summary>
    /// 获取网站名称
    /// </summary>
    /// <param name="strUrl"></param>
    /// <returns></returns>
    public static string GetWebUrl(string strUrl)
    {
        if (strUrl.IndexOf("http://") == 0)
            strUrl = strUrl.Substring(7, strUrl.Length - 7);
        string[] strItem = strUrl.Split('/');
        return strItem[0].ToString();
    }

    /// <summary>
    /// 根据网页的HTML内容提取网页的Encoding
    /// </summary>
    /// <param name="strHtml"></param>
    /// <returns></returns>
    public static Encoding GetEncoding(string strHtml)
    {
        string pattern = @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)";
        string charset = Regex.Match(strHtml, pattern).Groups["charset"].Value;
        try { return Encoding.GetEncoding(charset); }
        catch (ArgumentException) { return null; }
    }

    /// <summary>
    /// 根据网页的HTML内容提取网页的Title
    /// </summary>
    /// <param name="strUrl"></param>
    /// <returns></returns>
    public static string GetTitle(string strUrl)
    {
        string html = GetHtml(strUrl);
        string pattern = @"(?si)<title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(?<title>.*?)</title>";
        return Regex.Match(html, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline).Groups["title"].Value.Trim();
    }

    /// <summary>
    /// 根据网页的HTML内容提取网页的Title
    /// </summary>
    /// <param name="strUrl"></param>
    /// <param name="strHtml"></param>
    /// <returns></returns>
    public static string GetTitleAndHtml(string strUrl, ref string strHtml)
    {
        strHtml = GetHtml(strUrl);
        string pattern = @"(?si)<title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(?<title>.*?)</title>";
        return Regex.Match(strHtml, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline).Groups["title"].Value.Trim();
    }

    /// <summary>
    /// 根据网页的HTML内容提取网页的Title
    /// </summary>
    /// <param name="strUrl"></param>
    /// <param name="strHtml"></param>
    /// <returns></returns>
    public static string GetTitleAndBody(string strUrl, ref string strHtml)
    {
        strHtml = GetHtml(strUrl);
        string pattern = @"(?si)<title(?:\s+(?:""[^""]*""|'[^']*'|[^""'>])*)?>(?<title>.*?)</title>";
        string strTitle= Regex.Match(strHtml, pattern, RegexOptions.IgnoreCase | RegexOptions.Multiline).Groups["title"].Value.Trim();
        strHtml = NoHTML(strHtml);
        return strTitle;
    }

    /**/
    ///   <summary>
    ///   去除HTML标记
    ///   </summary>
    ///   <param   name="NoHTML">包括HTML的源码   </param>
    ///   <returns>已经去除后的文字</returns>
    public static string NoHTML(string strHTML)
    {
        //删除脚本
        strHTML = Regex.Replace(strHTML, @"<script[^>]*?>.*?</script>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        //删除HTML
        strHTML = Regex.Replace(strHTML, @"<(.[^>]*)>", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"([\r\n])[\s]+", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"-->", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"<!--.*", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(quot|#34);", "\"", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(amp|#38);", "&", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(lt|#60);", "<", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(gt|#62);", ">", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(nbsp|#160);", "   ", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(iexcl|#161);", "\xa1", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(cent|#162);", "\xa2", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(pound|#163);", "\xa3", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&ldquo;", "“", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&rdquo;", "”", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&(copy|#169);", "\xa9", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = Regex.Replace(strHTML, @"&#(\d+);", "", RegexOptions.IgnoreCase | RegexOptions.Multiline);
        strHTML = strHTML.Replace("\r\n", "");
        return strHTML;
    }

    /// <summary> 
    /// 将 Stream 转成 byte[] 
    /// </summary> 
    private static byte[] StreamToBytes(Stream stream)
    {
        MemoryStream stmMemory = new MemoryStream();
        byte[] buffer = new byte[4096];
        int i;
        while ((i = stream.Read(buffer, 0, buffer.Length)) > 0)
        {
            stmMemory.Write(buffer, 0, i);
        }
        byte[] arraryByte = stmMemory.ToArray();
        stmMemory.Close();


        // 设置当前流的位置为流的开始 
        if (stream.CanSeek)
        {
            stream.Seek(0, SeekOrigin.Begin);
        }
        return arraryByte;
    }
}